#!/usr/bin/python2.7
# -*- coding=utf-8 -*-
from bs4 import BeautifulSoup
import requests as req
import json
import io
import traceback
import time

baseUrl = "https://patents.google.com/patent/"
# "/opt/oldPatentJson/patent.json"
inputJsonPath = "/opt/oldPatentJson/2019/20191101.json"
# "/opt/newPatentJson/patent.json"
outputJsonPath = "/opt/newPatentJson/20191101.json"


# 解析网页，获取全文、权力要求书
def downloadPatentFullText(publish_no):
    res = req.get(baseUrl + publish_no)
    res.encoding = 'UTF-8'
    soup = BeautifulSoup(res.text, "html.parser")
    description = soup.find('section', attrs={"itemprop": "description"})
    if description is None:
        print('无数据:', publish_no)
    else:
        patentFullText = description.find('div').text
        if patentFullText is None:
            print('无数据:', publish_no)
        else:
            patentFullTextStr = "".join([line for line in patentFullText.split('\n') if line.strip() != ''])
    claim = soup.find('section', attrs={"itemprop": "claims"}).find('div').text
    if claim is None:
        print('无数据:', publish_no)
    else:
        claimStr = "".join([line for line in claim.split('\n') if line.strip() != ''])
    return [patentFullTextStr, claimStr]


# 批量写入
def batch_write(patent_list):
    with io.open(outputJsonPath, 'a', encoding="utf-8") as f:
        for patent in patent_list:
            f.write(json.dumps(patent, ensure_ascii=False) + '\n')


# 构造专利全文数据
def build_patent(line):
    data = json.loads(line)
    patent = {
        '_id': data['_id'],
        'pid': data['pid'],
        'appNo': data['appNo'],
        'publishNo': data['publishNo'],
        'publishDate': data['publishDate'],
        'patentType': data['patentType']
    }
    publishNo = data['publishNo']
    try:
        full_text = downloadPatentFullText(publishNo)
        time.sleep(1)
        patent['patentFullText'] = full_text[0]
        patent['patentClaim'] = full_text[1]
        patent['convertFlag'] = '0'
    except Exception as e:
        traceback.print_exc()
        patent['convertFlag'] = '1'
    return patent


def run():
    patent_list = []
    with io.open(inputJsonPath, 'r', encoding="utf-8") as f:
        line = f.readline()
        while line:
            patent_list.append(build_patent(line.strip()))
            if len(patent_list) == 100:
                batch_write(patent_list)
                patent_list = []
            line = f.readline()


if __name__ == '__main__':
    run()

